import json
from urllib.parse import urlencode

import requests
import time


def get_page_index(offset, keyword):
    # 请求时需要的参数
    data = {
        'aid': '24',
        'app_name': 'web_search',
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'en_qc': '1',
        'cur_tab': '1',
        'from': 'search_tab',
        'pd': 'synthesis',
        'timestamp': int(time.time()),
    }
    url = 'https://www.toutiao.com/api/search/content/?' + urlencode(data)
    headers = {
        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Mobile Safari/537.36",
        "cookie": "tt_webid=6818010969870059015; SLARDAR_WEB_ID=de0a6fa0-2353-4361-a091-f750c1d039b3; s_v_web_id=verify_k99dx1pd_hDVP0AHW_rkUL_4NpI_BR7H_CaUU49kDWHk1; WEATHER_CITY=%E5%8C%97%E4%BA%AC; tt_webid=6818010969870059015; csrftoken=e39f6a2faa3b2742b880f8939ab121b8; ttcid=3001d5b7eadf4314a152fb7c729be3cd13; __tasessionId=2ejuhcxwv1587452318019; tt_scid=cnsXgjICkjp-H2tEX2xRGjOHHlzEGHSM0xc-YgTdo.D-QsFU8aZ8YPHn0RcildVsfa8a"
    }
    # 发起请求
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return json.loads(response.text)
            # return response.text
    except:
        print('请求索引出错')
        return None


def parse_page_index(html):
    if html and 'data' in html.keys():
        for item in html.get('data'):
            # print(item)
            yield item.get('article_url')


def main():
    html = get_page_index(0, '街拍')
    # 获取详情页面的索引
    for url in parse_page_index(html):
        print(url)
    # print(html)


if __name__ == '__main__':
    main()
